result = query_api.query_data_frame(org=org, query=query)
result.columnsIndex(['result', 'table', '_start', '_stop', '_time', '_value', '_field',
'_measurement', 'device', 'sensor', 'units'],
dtype='object')
1 hour integral of power in kWh
2021/04/01 - 2023/11/09
Index(['result', 'table', '_start', '_stop', '_time', '_value', '_field',
'_measurement', 'device', 'sensor', 'units'],
dtype='object')
result.to_csv(f’./Data/energy_info.csv’, index=False) #save the data as a csv
| _time | _value | sensor | |
|---|---|---|---|
| 0 | 2023-03-02 00:00:00+00:00 | 12.888000 | Aircon |
| 1 | 2023-03-03 00:00:00+00:00 | 121.541007 | Aircon |
| 2 | 2023-03-04 00:00:00+00:00 | 222.993806 | Aircon |
| 3 | 2023-03-05 00:00:00+00:00 | 118.435757 | Aircon |
| 4 | 2023-03-06 00:00:00+00:00 | 238.721389 | Aircon |
wide_df = result.pivot(index='_time', columns='sensor', values='_value')
# Reset the index to make 'id' a regular column
wide_df.reset_index(inplace=True)
wide_df.columnsIndex(['_time', 'Aircon', 'Consumption', 'ConsumptionNet', 'Export', 'Fridge',
'Garage', 'Hotwater', 'HouseBoardOutside', 'HouseBoardOutsideR',
'HouseBoardR', 'Import', 'Lights', 'Net', 'OvenStove', 'Powerpoints',
'Production', 'Unmonitored'],
dtype='object', name='sensor')
wide_df_new = pd.DataFrame()
for column in wide_df.columns:
wide_df_new[f'{column}'] = wide_df[f'{column}'].interpolate()
wide_df = pd.DataFrame(wide_df_new)
wide_df.columnsIndex(['ds', 'Aircon', 'Consumption', 'ConsumptionNet', 'Export', 'Fridge',
'Garage', 'Hotwater', 'HouseBoardOutside', 'HouseBoardOutsideR',
'HouseBoardR', 'Import', 'Lights', 'Net', 'OvenStove', 'Powerpoints',
'Production'],
dtype='object')
| ds | Aircon | Consumption | ConsumptionNet | Export | Fridge | Garage | Hotwater | HouseBoardOutside | HouseBoardOutsideR | HouseBoardR | Import | Lights | Net | OvenStove | Powerpoints | Production | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 240 | 2023-10-28 | 8.578005 | 357.275503 | 866.654129 | 180.242247 | 25.492343 | 24.017087 | 0.530865 | 999.775552 | 333.258545 | 366.408449 | 71.876417 | 80.009421 | -108.365830 | 66.086281 | 176.392548 | 491.748285 |
| 241 | 2023-10-29 | 8.661682 | 179.501779 | 866.654129 | 806.610093 | 19.634587 | 2.731102 | 0.192283 | 530.312474 | 176.770805 | 201.397413 | 1.974071 | 60.108551 | -804.636021 | 1.253300 | 103.451046 | 1017.375874 |
| 242 | 2023-10-30 | 8.603354 | 263.812291 | 866.654129 | 757.601472 | 27.081000 | 0.983716 | 0.183639 | 788.486015 | 262.828480 | 290.616882 | 3.378339 | 66.827993 | -754.223132 | 1.356785 | 178.170521 | 1052.272015 |
| 243 | 2023-10-31 | 9.032917 | 311.552394 | 866.654129 | 803.841970 | 35.649472 | 11.163953 | 0.269313 | 901.165305 | 300.388379 | 319.795146 | 5.027126 | 67.972347 | -798.814844 | 30.217549 | 166.649785 | 1142.661342 |
| 244 | 2023-11-01 | 8.743938 | 211.339011 | 866.654129 | 898.594991 | 30.389375 | 3.655479 | 0.189410 | 623.050504 | 207.683462 | 245.486118 | 2.309883 | 50.624125 | -896.285107 | 0.213375 | 145.941312 | 1134.978184 |
def long_form(df_plot):
return df_plot.melt('ds', var_name='unique_id', value_name='y')
def altair_plot(df_plot):
import altair as alt
highlight = alt.selection_point(on='mouseover', fields=['unique_id'], bind='legend', nearest=True)
base = alt.Chart(df_plot).encode(
x='ds:T',
y='y:Q',
color='unique_id:N'
)
points = base.mark_circle().encode(
opacity=alt.value(0)
).add_params(
highlight
).properties(
width=1000
)
lines = base.mark_line().encode(
size=alt.condition(~highlight, alt.value(1), alt.value(3))
)
return points + lines| ds | Aircon | Consumption | ConsumptionNet | Export | Fridge | Garage | Hotwater | HouseBoardOutside | HouseBoardOutsideR | HouseBoardR | Import | Lights | Net | OvenStove | Powerpoints | Production | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 240 | 2023-10-28 | 0.000327 | 0.295413 | 0.639383 | 0.087269 | 0.054518 | 0.475787 | 0.000988 | 0.441317 | 0.270647 | 0.263152 | 0.098826 | 0.397064 | 0.702175 | 0.217335 | 0.451213 | 0.164710 |
| 241 | 2023-10-29 | 0.000459 | 0.096052 | 0.639383 | 0.390543 | 0.008991 | 0.049113 | 0.000167 | 0.155476 | 0.095349 | 0.096613 | 0.001842 | 0.287229 | 0.452268 | 0.003559 | 0.195780 | 0.385757 |
| 242 | 2023-10-30 | 0.000367 | 0.190600 | 0.639383 | 0.366814 | 0.066865 | 0.014087 | 0.000146 | 0.312670 | 0.191751 | 0.186659 | 0.003790 | 0.324314 | 0.470363 | 0.003900 | 0.457440 | 0.400433 |
| 243 | 2023-10-31 | 0.001045 | 0.244137 | 0.639383 | 0.389203 | 0.133461 | 0.218148 | 0.000354 | 0.381277 | 0.233826 | 0.216107 | 0.006078 | 0.330630 | 0.454358 | 0.099064 | 0.417095 | 0.438445 |
| 244 | 2023-11-01 | 0.000589 | 0.131755 | 0.639383 | 0.435080 | 0.092579 | 0.067642 | 0.000160 | 0.211942 | 0.129978 | 0.141110 | 0.002308 | 0.234883 | 0.419373 | 0.000130 | 0.344577 | 0.435214 |
import pandas as pd
path = 'Data'
name = 'weather'
weather_df = pd.read_csv(f'{path}/{name}.csv')
weather_df.head()| ds | temp_c | is_day | wind_kph | wind_degree | pressure_mb | precip_mm | humidity | cloud | feelslike_c | windchill_c | heatindex_c | dewpoint_c | chance_of_rain | vis_km | gust_kph | uv | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 2023-01-01 00:00 | 21.6 | 0 | 12.6 | 126 | 1013.0 | 0.0 | 84 | 73 | 21.6 | 21.6 | 24.4 | 18.7 | 0 | 10.0 | 19.8 | 1.0 |
| 1 | 2023-01-01 01:00 | 21.3 | 0 | 11.9 | 133 | 1013.0 | 0.0 | 85 | 77 | 21.3 | 21.3 | 24.3 | 18.7 | 0 | 10.0 | 18.7 | 1.0 |
| 2 | 2023-01-01 02:00 | 21.1 | 0 | 11.2 | 140 | 1012.0 | 0.0 | 86 | 80 | 21.1 | 21.1 | 24.2 | 18.7 | 0 | 10.0 | 17.6 | 1.0 |
| 3 | 2023-01-01 03:00 | 20.8 | 0 | 10.4 | 147 | 1012.0 | 0.0 | 88 | 84 | 20.8 | 20.8 | 20.8 | 18.7 | 0 | 10.0 | 16.6 | 1.0 |
| 4 | 2023-01-01 04:00 | 21.0 | 0 | 10.6 | 144 | 1012.0 | 0.0 | 86 | 84 | 21.0 | 21.0 | 21.0 | 18.6 | 0 | 10.0 | 16.3 | 1.0 |
def data_day_avg(weather_df):
from tqdm.notebook import tqdm
avg_df = pd.DataFrame()
dates = pd.to_datetime(weather_df['ds']).dt.date.unique()
for date in tqdm(dates):
filtered_df = weather_df[pd.to_datetime(weather_df['ds']).dt.date == date]
ds = filtered_df.pop('ds')
filtered_df = pd.DataFrame(filtered_df.mean()).T
filtered_df
filtered_df.insert(0, 'ds', date)
avg_df = pd.concat([avg_df, filtered_df], ignore_index=True)
return avg_df| ds | temp_c | is_day | wind_kph | wind_degree | pressure_mb | precip_mm | humidity | cloud | feelslike_c | windchill_c | heatindex_c | dewpoint_c | chance_of_rain | vis_km | gust_kph | uv | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 2023-01-01 | 0.566503 | 0.583333 | 0.459098 | 0.345636 | 0.428728 | 0.000895 | 0.670886 | 0.583750 | 0.516791 | 0.595085 | 0.512881 | 0.778121 | 0.0 | 1.000000 | 0.434678 | 0.422619 |
| 1 | 2023-01-02 | 0.554902 | 0.583333 | 0.451453 | 0.396936 | 0.359649 | 0.001253 | 0.602848 | 0.409167 | 0.499005 | 0.584249 | 0.486067 | 0.725707 | 0.0 | 1.000000 | 0.434844 | 0.428571 |
| 2 | 2023-01-03 | 0.582516 | 0.583333 | 0.263889 | 0.415506 | 0.258772 | 0.000000 | 0.584916 | 0.287917 | 0.525746 | 0.610043 | 0.502366 | 0.738663 | 0.0 | 1.000000 | 0.283533 | 0.452381 |
| 3 | 2023-01-04 | 0.650490 | 0.583333 | 0.313073 | 0.382660 | 0.163377 | 0.000895 | 0.687764 | 0.196250 | 0.608582 | 0.673535 | 0.609621 | 0.866902 | 0.0 | 0.994792 | 0.305777 | 0.500000 |
| 4 | 2023-01-05 | 0.654575 | 0.583333 | 0.257518 | 0.506267 | 0.166667 | 0.026859 | 0.735232 | 0.700417 | 0.629726 | 0.677350 | 0.608701 | 0.911808 | 0.0 | 0.989583 | 0.288098 | 0.416667 |
| ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
| 339 | 2023-12-06 | 0.580556 | 0.583333 | 0.254077 | 0.264740 | 0.558114 | 0.000000 | 0.608650 | 0.174583 | 0.519527 | 0.608364 | 0.515116 | 0.745878 | 0.0 | 1.000000 | 0.214558 | 0.470238 |
| 340 | 2023-12-07 | 0.605065 | 0.583333 | 0.266565 | 0.202066 | 0.565789 | 0.000000 | 0.582806 | 0.050000 | 0.543532 | 0.631258 | 0.533386 | 0.748233 | 0.0 | 1.000000 | 0.222693 | 0.476190 |
| 341 | 2023-12-08 | 0.618954 | 0.583333 | 0.349261 | 0.251625 | 0.575658 | 0.000018 | 0.626582 | 0.342083 | 0.561816 | 0.644078 | 0.555205 | 0.795200 | 0.0 | 1.000000 | 0.285940 | 0.476190 |
| 342 | 2023-12-09 | 0.612582 | 0.583333 | 0.362768 | 0.190460 | 0.544956 | 0.000107 | 0.577532 | 0.497500 | 0.564179 | 0.638584 | 0.539432 | 0.757509 | 0.0 | 1.000000 | 0.308267 | 0.428571 |
| 343 | 2023-12-10 | 0.630556 | 0.583333 | 0.513507 | 0.233751 | 0.530702 | 0.000233 | 0.515295 | 0.537500 | 0.571269 | 0.655067 | 0.546924 | 0.730271 | 0.0 | 1.000000 | 0.426627 | 0.452381 |
344 rows × 17 columns
/home/ben/mambaforge/envs/cfast/lib/python3.11/site-packages/statsforecast/core.py:25: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)
from tqdm.autonotebook import tqdm
from pylab import rcParams
rcParams['figure.figsize'] = (15,4)
StatsForecast.plot(df, engine='plotly')/home/ben/mambaforge/envs/cfast/lib/python3.11/site-packages/_plotly_utils/basevalidators.py:105: FutureWarning:
The behavior of DatetimeProperties.to_pydatetime is deprecated, in a future version this will return a Series containing python datetime objects instead of an ndarray. To retain the old behavior, call `np.array` on the result
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 245 entries, 0 to 244
Data columns (total 18 columns):
# Column Non-Null Count Dtype
--- ------ -------------- -----
0 ds 245 non-null datetime64[ns]
1 Aircon 245 non-null float64
2 y 245 non-null float64
3 ConsumptionNet 198 non-null float64
4 Export 245 non-null float64
5 Fridge 245 non-null float64
6 Garage 245 non-null float64
7 Hotwater 245 non-null float64
8 HouseBoardOutside 161 non-null float64
9 HouseBoardOutsideR 245 non-null float64
10 HouseBoardR 245 non-null float64
11 Import 245 non-null float64
12 Lights 245 non-null float64
13 Net 245 non-null float64
14 OvenStove 245 non-null float64
15 Powerpoints 245 non-null float64
16 Production 245 non-null float64
17 unique_id 245 non-null object
dtypes: datetime64[ns](1), float64(16), object(1)
memory usage: 34.6+ KB
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 3920 entries, 0 to 3919
Data columns (total 3 columns):
# Column Non-Null Count Dtype
--- ------ -------------- -----
0 ds 3920 non-null datetime64[ns]
1 unique_id 3920 non-null object
2 y 3789 non-null float64
dtypes: datetime64[ns](1), float64(1), object(1)
memory usage: 92.0+ KB
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 344 entries, 0 to 343
Data columns (total 17 columns):
# Column Non-Null Count Dtype
--- ------ -------------- -----
0 ds 344 non-null datetime64[ns]
1 temp_c 344 non-null float64
2 is_day 344 non-null float64
3 wind_kph 344 non-null float64
4 wind_degree 344 non-null float64
5 pressure_mb 344 non-null float64
6 precip_mm 344 non-null float64
7 humidity 344 non-null float64
8 cloud 344 non-null float64
9 feelslike_c 344 non-null float64
10 windchill_c 344 non-null float64
11 heatindex_c 344 non-null float64
12 dewpoint_c 344 non-null float64
13 chance_of_rain 344 non-null float64
14 vis_km 344 non-null float64
15 gust_kph 344 non-null float64
16 uv 344 non-null float64
dtypes: datetime64[ns](1), float64(16)
memory usage: 45.8 KB
date_range_start, date_range_end = wide_df['ds'].min(), wide_df['ds'].max()
# Filter rows between the start and end dates
weather_avg_filtered_df = weather_avg_df[(weather_avg_df['ds'] >= date_range_start) & (weather_avg_df['ds'] <= date_range_end)]
weather_avg_filtered_df| ds | temp_c | is_day | wind_kph | wind_degree | pressure_mb | precip_mm | humidity | cloud | feelslike_c | windchill_c | heatindex_c | dewpoint_c | chance_of_rain | vis_km | gust_kph | uv | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 60 | 2023-03-02 | 0.644444 | 0.541667 | 0.294980 | 0.153784 | 0.344298 | 0.001056 | 0.675633 | 0.345417 | 0.609826 | 0.667888 | 0.594374 | 0.845112 | 0.0 | 1.0 | 0.250332 | 0.470238 |
| 61 | 2023-03-03 | 0.603268 | 0.541667 | 0.483563 | 0.386142 | 0.403509 | 0.002131 | 0.682489 | 0.579167 | 0.572388 | 0.629426 | 0.548107 | 0.821555 | 0.0 | 1.0 | 0.432022 | 0.380952 |
| 62 | 2023-03-04 | 0.561765 | 0.541667 | 0.574669 | 0.433612 | 0.425439 | 0.000161 | 0.604430 | 0.364583 | 0.509204 | 0.590659 | 0.491982 | 0.734982 | 0.0 | 1.0 | 0.485309 | 0.422619 |
| 63 | 2023-03-05 | 0.598039 | 0.541667 | 0.372324 | 0.350627 | 0.427632 | 0.000358 | 0.603903 | 0.401250 | 0.542537 | 0.624542 | 0.531677 | 0.761042 | 0.0 | 1.0 | 0.319804 | 0.428571 |
| 64 | 2023-03-06 | 0.629575 | 0.541667 | 0.313073 | 0.290854 | 0.417763 | 0.000358 | 0.608650 | 0.157500 | 0.568284 | 0.653999 | 0.557965 | 0.785925 | 0.0 | 1.0 | 0.263363 | 0.476190 |
| ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |
| 300 | 2023-10-28 | 0.379575 | 0.541667 | 0.700051 | 0.446611 | 0.708333 | 0.000233 | 0.500000 | 0.389167 | 0.351741 | 0.414835 | 0.327681 | 0.497939 | 0.0 | 1.0 | 0.588811 | 0.345238 |
| 301 | 2023-10-29 | 0.382516 | 0.541667 | 0.383028 | 0.320682 | 0.656798 | 0.000125 | 0.524789 | 0.433750 | 0.347637 | 0.423230 | 0.329390 | 0.525324 | 0.0 | 1.0 | 0.328934 | 0.345238 |
| 302 | 2023-10-30 | 0.441340 | 0.583333 | 0.396152 | 0.435237 | 0.562500 | 0.000000 | 0.583861 | 0.181250 | 0.403483 | 0.478327 | 0.373686 | 0.611307 | 0.0 | 1.0 | 0.340139 | 0.416667 |
| 303 | 2023-10-31 | 0.550163 | 0.583333 | 0.489424 | 0.549559 | 0.442982 | 0.000000 | 0.570675 | 0.025417 | 0.492537 | 0.579976 | 0.467140 | 0.687132 | 0.0 | 1.0 | 0.470286 | 0.464286 |
| 304 | 2023-11-01 | 0.510784 | 0.583333 | 0.561672 | 0.445218 | 0.505482 | 0.000340 | 0.599156 | 0.480000 | 0.458209 | 0.543040 | 0.435594 | 0.685218 | 0.0 | 1.0 | 0.467795 | 0.404762 |
245 rows × 17 columns
0 0.355157
1 0.291076
2 0.498418
3 0.418824
4 0.561599
...
240 0.295413
241 0.096052
242 0.190600
243 0.244137
244 0.131755
Name: y, Length: 245, dtype: float64
fig, axs = plt.subplots(nrows=1, ncols=2)
plot_acf(wide_df["y"], lags=28, ax=axs[0],color="fuchsia")
axs[0].set_title("Autocorrelation");
plot_pacf(wide_df["y"], lags=28, ax=axs[1],color="lime")
axs[1].set_title('Partial Autocorrelation')
plt.show();
| ds | temp_c | is_day | wind_kph | wind_degree | pressure_mb | precip_mm | humidity | cloud | feelslike_c | windchill_c | heatindex_c | dewpoint_c | chance_of_rain | vis_km | gust_kph | uv | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 60 | 2023-03-02 | 0.644444 | 0.541667 | 0.294980 | 0.153784 | 0.344298 | 0.001056 | 0.675633 | 0.345417 | 0.609826 | 0.667888 | 0.594374 | 0.845112 | 0.0 | 1.0 | 0.250332 | 0.470238 |
| 61 | 2023-03-03 | 0.603268 | 0.541667 | 0.483563 | 0.386142 | 0.403509 | 0.002131 | 0.682489 | 0.579167 | 0.572388 | 0.629426 | 0.548107 | 0.821555 | 0.0 | 1.0 | 0.432022 | 0.380952 |
| 62 | 2023-03-04 | 0.561765 | 0.541667 | 0.574669 | 0.433612 | 0.425439 | 0.000161 | 0.604430 | 0.364583 | 0.509204 | 0.590659 | 0.491982 | 0.734982 | 0.0 | 1.0 | 0.485309 | 0.422619 |
| 63 | 2023-03-05 | 0.598039 | 0.541667 | 0.372324 | 0.350627 | 0.427632 | 0.000358 | 0.603903 | 0.401250 | 0.542537 | 0.624542 | 0.531677 | 0.761042 | 0.0 | 1.0 | 0.319804 | 0.428571 |
| 64 | 2023-03-06 | 0.629575 | 0.541667 | 0.313073 | 0.290854 | 0.417763 | 0.000358 | 0.608650 | 0.157500 | 0.568284 | 0.653999 | 0.557965 | 0.785925 | 0.0 | 1.0 | 0.263363 | 0.476190 |
| ds | Aircon | y | ConsumptionNet | Export | Fridge | Garage | Hotwater | HouseBoardOutside | HouseBoardOutsideR | ... | humidity | cloud | feelslike_c | windchill_c | heatindex_c | dewpoint_c | chance_of_rain | vis_km | gust_kph | uv | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 2023-03-02 | 0.007134 | 0.355157 | NaN | 0.016595 | 0.814609 | 0.003794 | 0.000287 | NaN | 0.356702 | ... | 0.675633 | 0.345417 | 0.609826 | 0.667888 | 0.594374 | 0.845112 | 0.0 | 1.0 | 0.250332 | 0.470238 |
| 1 | 2023-03-03 | 0.178748 | 0.291076 | NaN | 0.165011 | 0.669709 | 0.015031 | 0.000143 | NaN | 0.292065 | ... | 0.682489 | 0.579167 | 0.572388 | 0.629426 | 0.548107 | 0.821555 | 0.0 | 1.0 | 0.432022 | 0.380952 |
| 2 | 2023-03-04 | 0.338990 | 0.498418 | NaN | 0.294620 | 0.686509 | 0.059559 | 0.000460 | NaN | 0.496691 | ... | 0.604430 | 0.364583 | 0.509204 | 0.590659 | 0.491982 | 0.734982 | 0.0 | 1.0 | 0.485309 | 0.422619 |
| 3 | 2023-03-05 | 0.173844 | 0.418824 | NaN | 0.302607 | 0.701288 | 0.009916 | 0.000256 | NaN | 0.419958 | ... | 0.603903 | 0.401250 | 0.542537 | 0.624542 | 0.531677 | 0.761042 | 0.0 | 1.0 | 0.319804 | 0.428571 |
| 4 | 2023-03-06 | 0.363831 | 0.561599 | NaN | 0.381220 | 0.773714 | 0.008798 | 0.002112 | NaN | 0.562640 | ... | 0.608650 | 0.157500 | 0.568284 | 0.653999 | 0.557965 | 0.785925 | 0.0 | 1.0 | 0.263363 | 0.476190 |
5 rows × 34 columns
horizon = 10
train_size = len(wide_df) - horizon
train, test = wide_df[:train_size], wide_df[train_size:]
wide_df.shape, train.shape, test.shape((245, 34), (235, 34), (10, 34))
/tmp/ipykernel_1270281/3311167277.py:2: SettingWithCopyWarning:
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead
See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
(10, 18)
| ds | temp_c | is_day | wind_kph | wind_degree | pressure_mb | precip_mm | humidity | cloud | feelslike_c | windchill_c | heatindex_c | dewpoint_c | chance_of_rain | vis_km | gust_kph | uv | unique_id | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 295 | 2023-10-23 | 0.560784 | 0.500000 | 0.579256 | 0.550836 | 0.439693 | 0.000000 | 0.614979 | 0.075000 | 0.509328 | 0.589896 | 0.485279 | 0.734246 | 0.0 | 1.000000 | 0.512699 | 0.392857 | Consumption |
| 296 | 2023-10-24 | 0.550490 | 0.541667 | 0.571738 | 0.437210 | 0.500000 | 0.000430 | 0.684599 | 0.150833 | 0.503483 | 0.580128 | 0.486593 | 0.771349 | 0.0 | 1.000000 | 0.495933 | 0.410714 | Consumption |
| 297 | 2023-10-25 | 0.558660 | 0.541667 | 0.585117 | 0.546657 | 0.482456 | 0.000000 | 0.656646 | 0.073750 | 0.511070 | 0.588217 | 0.487119 | 0.759570 | 0.0 | 1.000000 | 0.513197 | 0.428571 | Consumption |
| 298 | 2023-10-26 | 0.495261 | 0.541667 | 0.518349 | 0.551996 | 0.493421 | 0.005766 | 0.684072 | 0.593333 | 0.449627 | 0.528541 | 0.422581 | 0.725265 | 0.0 | 0.953125 | 0.448705 | 0.363095 | Consumption |
| 299 | 2023-10-27 | 0.309314 | 0.541667 | 0.621814 | 0.516481 | 0.633772 | 0.008792 | 0.767932 | 0.862500 | 0.289552 | 0.355311 | 0.249211 | 0.616166 | 0.0 | 0.994792 | 0.573373 | 0.232143 | Consumption |
| 300 | 2023-10-28 | 0.379575 | 0.541667 | 0.700051 | 0.446611 | 0.708333 | 0.000233 | 0.500000 | 0.389167 | 0.351741 | 0.414835 | 0.327681 | 0.497939 | 0.0 | 1.000000 | 0.588811 | 0.345238 | Consumption |
| 301 | 2023-10-29 | 0.382516 | 0.541667 | 0.383028 | 0.320682 | 0.656798 | 0.000125 | 0.524789 | 0.433750 | 0.347637 | 0.423230 | 0.329390 | 0.525324 | 0.0 | 1.000000 | 0.328934 | 0.345238 | Consumption |
| 302 | 2023-10-30 | 0.441340 | 0.583333 | 0.396152 | 0.435237 | 0.562500 | 0.000000 | 0.583861 | 0.181250 | 0.403483 | 0.478327 | 0.373686 | 0.611307 | 0.0 | 1.000000 | 0.340139 | 0.416667 | Consumption |
| 303 | 2023-10-31 | 0.550163 | 0.583333 | 0.489424 | 0.549559 | 0.442982 | 0.000000 | 0.570675 | 0.025417 | 0.492537 | 0.579976 | 0.467140 | 0.687132 | 0.0 | 1.000000 | 0.470286 | 0.464286 | Consumption |
| 304 | 2023-11-01 | 0.510784 | 0.583333 | 0.561672 | 0.445218 | 0.505482 | 0.000340 | 0.599156 | 0.480000 | 0.458209 | 0.543040 | 0.435594 | 0.685218 | 0.0 | 1.000000 | 0.467795 | 0.404762 | Consumption |
['RNN',
'GRU',
'LSTM',
'TCN',
'DeepAR',
'DilatedRNN',
'MLP',
'NHITS',
'NBEATS',
'NBEATSx',
'TFT',
'VanillaTransformer',
'Informer',
'Autoformer',
'PatchTST',
'FEDformer',
'StemGNN',
'HINT',
'TimesNet']
model_list = [RNN,
GRU,
LSTM,
TCN,
#DeepAR, # not good
DilatedRNN,
MLP,
NHITS,
NBEATS,
NBEATSx,
#TFT, # too much GPU
#VanillaTransformer, # not good
#Informer,
#Autoformer,
#PatchTST, # too much GPU
#FEDformer, #taken too long
#StemGNN, #need n_series
#HINT, #need n_series
#TimesNet # takes too long
]Index(['ds', 'Aircon', 'y', 'ConsumptionNet', 'Export', 'Fridge', 'Garage',
'Hotwater', 'HouseBoardOutside', 'HouseBoardOutsideR', 'HouseBoardR',
'Import', 'Lights', 'Net', 'OvenStove', 'Powerpoints', 'Production',
'unique_id', 'temp_c', 'is_day', 'wind_kph', 'wind_degree',
'pressure_mb', 'precip_mm', 'humidity', 'cloud', 'feelslike_c',
'windchill_c', 'heatindex_c', 'dewpoint_c', 'chance_of_rain', 'vis_km',
'gust_kph', 'uv'],
dtype='object')
models = [model(input_size=2 * horizon,
h=horizon,
max_steps=500,
hist_exog_list =['Aircon', 'Export', 'Fridge', 'Garage',
'Hotwater',
'HouseBoardR','Import', 'Lights', 'Net', 'OvenStove',
'Powerpoints', 'Production'
],
futr_exog_list = ['temp_c', 'is_day', 'wind_kph', 'wind_degree',
'pressure_mb', 'precip_mm', 'humidity', 'cloud', 'feelslike_c',
'windchill_c', 'heatindex_c', 'dewpoint_c', 'chance_of_rain',
'vis_km', 'gust_kph', 'uv']
) for model in model_list]
modelsGlobal seed set to 1
Global seed set to 1
Global seed set to 1
Global seed set to 1
Global seed set to 1
Global seed set to 1
Global seed set to 1
Global seed set to 1
Global seed set to 1
[RNN, GRU, LSTM, TCN, DilatedRNN, MLP, NHITS, NBEATS, NBEATSx]
Index(['ds', 'Aircon', 'y', 'ConsumptionNet', 'Export', 'Fridge', 'Garage',
'Hotwater', 'HouseBoardOutside', 'HouseBoardOutsideR', 'HouseBoardR',
'Import', 'Lights', 'Net', 'OvenStove', 'Powerpoints', 'Production',
'unique_id', 'temp_c', 'is_day', 'wind_kph', 'wind_degree',
'pressure_mb', 'precip_mm', 'humidity', 'cloud', 'feelslike_c',
'windchill_c', 'heatindex_c', 'dewpoint_c', 'chance_of_rain', 'vis_km',
'gust_kph', 'uv'],
dtype='object')
| unique_id | ds | RNN | GRU | LSTM | TCN | DilatedRNN | MLP | NHITS | NBEATS | NBEATSx | |
|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | Consumption | 2023-10-23 | 0.187143 | 0.340722 | 0.270740 | 0.227037 | 0.262816 | 0.142264 | 0.266232 | 0.310167 | 0.164331 |
| 1 | Consumption | 2023-10-24 | 0.120339 | 0.247064 | 0.097423 | 0.069222 | 0.166623 | 0.137484 | 0.146339 | 0.264303 | 0.187077 |
| 2 | Consumption | 2023-10-25 | 0.106269 | 0.185992 | 0.218475 | 0.100015 | 0.174694 | 0.203210 | 0.233841 | 0.362458 | 0.266043 |
| 3 | Consumption | 2023-10-26 | 0.037814 | 0.153535 | 0.145480 | 0.042539 | 0.125000 | 0.265789 | 0.335360 | 0.289754 | 0.315103 |
| 4 | Consumption | 2023-10-27 | 0.021276 | 0.049343 | -0.004718 | 0.034827 | 0.095918 | 0.238553 | 0.303091 | 0.309346 | 0.279027 |
consumption_predict_plot = long_form(consumption_predict)
consumption_predict_plot = pd.concat([consumption_predict_plot, just_consumption]
, ignore_index=True)
consumption_predict_plot| ds | unique_id | y | |
|---|---|---|---|
| 0 | 2023-10-23 | RNN | 0.187143 |
| 1 | 2023-10-24 | RNN | 0.120339 |
| 2 | 2023-10-25 | RNN | 0.106269 |
| 3 | 2023-10-26 | RNN | 0.037814 |
| 4 | 2023-10-27 | RNN | 0.021276 |
| ... | ... | ... | ... |
| 330 | 2023-10-28 | Consumption | 0.295413 |
| 331 | 2023-10-29 | Consumption | 0.096052 |
| 332 | 2023-10-30 | Consumption | 0.190600 |
| 333 | 2023-10-31 | Consumption | 0.244137 |
| 334 | 2023-11-01 | Consumption | 0.131755 |
335 rows × 3 columns
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 10 entries, 0 to 9
Data columns (total 10 columns):
# Column Non-Null Count Dtype
--- ------ -------------- -----
0 ds 10 non-null datetime64[ns]
1 RNN 10 non-null float64
2 GRU 10 non-null float64
3 LSTM 10 non-null float64
4 TCN 10 non-null float64
5 DilatedRNN 10 non-null float64
6 MLP 10 non-null float64
7 NHITS 10 non-null float64
8 NBEATS 10 non-null float64
9 NBEATSx 10 non-null float64
dtypes: datetime64[ns](1), float64(9)
memory usage: 932.0 bytes
(None,
Index(['ds', 'RNN', 'GRU', 'LSTM', 'TCN', 'DilatedRNN', 'MLP', 'NHITS',
'NBEATS', 'NBEATSx'],
dtype='object'))
def evaluate_performace(y_hist, y_true, y_pred):
#y_true = y_true.merge(y_pred, how='left', on=['unique_id', 'ds'])
evaluation = {}
print(y_pred.columns)
for i, model in enumerate(y_pred.columns):
if i < 1: continue
evaluation[model] = {}
for metric in [mase, mae, mape, rmse, smape]:
metric_name = metric.__name__
if metric_name == 'mase':
evaluation[model][metric_name] = metric(y_true['y'].values,
y_pred[model].values,
y_hist['y'].values, seasonality=12)
else:
evaluation[model][metric_name] = metric(y_true['y'].values, y_pred[model].values)
return pd.DataFrame(evaluation)score = evaluate_performace(train, test, consumption_predict)
score.style.background_gradient(cmap ='YlGn', axis = 1,low=0.5, high=0.6)Index(['ds', 'RNN', 'GRU', 'LSTM', 'TCN', 'DilatedRNN', 'MLP', 'NHITS',
'NBEATS', 'NBEATSx'],
dtype='object')
| RNN | GRU | LSTM | TCN | DilatedRNN | MLP | NHITS | NBEATS | NBEATSx | |
|---|---|---|---|---|---|---|---|---|---|
| mase | 0.827192 | 0.475914 | 0.581997 | 0.854392 | 0.532540 | 0.891489 | 1.069189 | 1.064738 | 1.062052 |
| mae | 0.124269 | 0.071497 | 0.087433 | 0.128355 | 0.080004 | 0.133928 | 0.160624 | 0.159956 | 0.159552 |
| mape | 55.463221 | 31.560946 | 31.207467 | 48.447932 | 29.925153 | 68.742079 | 92.780892 | 92.143272 | 92.818673 |
| rmse | 0.164423 | 0.107169 | 0.156157 | 0.176625 | 0.128558 | 0.170495 | 0.195339 | 0.177750 | 0.192040 |
| smape | 76.016902 | 34.840016 | 47.025939 | 71.174894 | 34.164025 | 55.432298 | 60.084229 | 58.849896 | 60.377130 |
Index(['unique_id', 'ds', 'cutoff', 'RNN', 'GRU', 'LSTM', 'TCN', 'DilatedRNN',
'MLP', 'NHITS', 'NBEATS', 'NBEATSx', 'Aircon', 'y', 'ConsumptionNet',
'Export', 'Fridge', 'Garage', 'Hotwater', 'HouseBoardOutside',
'HouseBoardOutsideR', 'HouseBoardR', 'Import', 'Lights', 'Net',
'OvenStove', 'Powerpoints', 'Production', 'temp_c', 'is_day',
'wind_kph', 'wind_degree', 'pressure_mb', 'precip_mm', 'humidity',
'cloud', 'feelslike_c', 'windchill_c', 'heatindex_c', 'dewpoint_c',
'chance_of_rain', 'vis_km', 'gust_kph', 'uv'],
dtype='object')
0 0.135751
1 0.069040
2 0.035342
3 0.191986
4 0.166608
5 0.172630
6 0.276067
7 0.224828
8 0.119483
9 0.142353
10 0.102038
11 0.065945
12 0.183238
13 0.203040
14 0.190365
15 0.209685
16 0.239724
17 0.131889
18 0.137529
19 0.309497
20 0.131816
21 0.200610
22 0.181522
23 0.179420
24 0.192814
25 0.191258
26 0.125263
27 0.109986
28 0.267251
29 0.237799
Name: RNN, dtype: float32
Index(['ds', 'RNN', 'GRU', 'LSTM', 'TCN', 'DilatedRNN', 'MLP', 'NHITS',
'NBEATS', 'NBEATSx'],
dtype='object')
| ds | RNN | GRU | LSTM | TCN | DilatedRNN | MLP | NHITS | NBEATS | NBEATSx | |
|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 2023-10-11 | 0.135751 | 0.067367 | 0.110072 | 0.087748 | 0.100938 | 0.102914 | 0.160495 | 0.176175 | 0.128718 |
| 1 | 2023-10-12 | 0.069040 | 0.149712 | 0.197463 | 0.148317 | 0.126475 | 0.112232 | 0.078474 | 0.145379 | 0.088011 |
| 2 | 2023-10-13 | 0.035342 | 0.187814 | 0.194257 | 0.140832 | 0.177151 | 0.041507 | 0.077609 | 0.112854 | 0.052702 |
| 3 | 2023-10-14 | 0.191986 | 0.232270 | 0.253045 | 0.228376 | 0.199984 | 0.159770 | 0.178985 | 0.242857 | 0.067291 |
| 4 | 2023-10-15 | 0.166608 | 0.128854 | 0.169904 | 0.126402 | 0.164642 | 0.215704 | 0.143691 | 0.212753 | 0.167671 |
| 5 | 2023-10-16 | 0.172630 | 0.208335 | 0.227233 | 0.223335 | 0.202316 | 0.139277 | 0.256585 | 0.268092 | 0.178975 |
| 6 | 2023-10-17 | 0.276067 | 0.281430 | 0.284902 | 0.354908 | 0.246623 | 0.305722 | 0.252022 | 0.270245 | 0.243796 |
| 7 | 2023-10-18 | 0.224828 | 0.219993 | 0.226789 | 0.329371 | 0.222550 | 0.262789 | 0.249297 | 0.274197 | 0.279998 |
| 8 | 2023-10-19 | 0.119483 | 0.204906 | 0.162221 | 0.165711 | 0.210179 | 0.112409 | 0.100446 | 0.252383 | 0.193922 |
| 9 | 2023-10-20 | 0.142353 | 0.226974 | 0.191427 | 0.111255 | 0.154943 | 0.302589 | 0.209240 | 0.218997 | 0.152117 |
| 10 | 2023-10-12 | 0.102038 | 0.111689 | 0.155506 | 0.097696 | 0.139405 | 0.065505 | 0.063259 | 0.111341 | 0.082719 |
| 11 | 2023-10-13 | 0.065945 | 0.173949 | 0.210845 | 0.121179 | 0.138233 | 0.151942 | 0.068114 | 0.124901 | 0.112885 |
| 12 | 2023-10-14 | 0.183238 | 0.209708 | 0.224227 | 0.221133 | 0.225971 | 0.141812 | 0.198544 | 0.221697 | 0.127999 |
| 13 | 2023-10-15 | 0.203040 | 0.189177 | 0.187111 | 0.244232 | 0.242181 | 0.160846 | 0.196839 | 0.221572 | 0.114414 |
| 14 | 2023-10-16 | 0.190365 | 0.249911 | 0.216426 | 0.253370 | 0.215205 | 0.200827 | 0.154640 | 0.255481 | 0.214551 |
| 15 | 2023-10-17 | 0.209685 | 0.212194 | 0.246824 | 0.305890 | 0.202765 | 0.186043 | 0.265844 | 0.213276 | 0.161825 |
| 16 | 2023-10-18 | 0.239724 | 0.239608 | 0.244121 | 0.271035 | 0.233068 | 0.306250 | 0.258346 | 0.316249 | 0.219644 |
| 17 | 2023-10-19 | 0.131889 | 0.198312 | 0.168622 | 0.159480 | 0.207734 | 0.131938 | 0.148374 | 0.250731 | 0.293880 |
| 18 | 2023-10-20 | 0.137529 | 0.205980 | 0.201126 | 0.135570 | 0.194409 | 0.162717 | 0.189692 | 0.192537 | 0.217399 |
| 19 | 2023-10-21 | 0.309497 | 0.257568 | 0.222876 | 0.194708 | 0.225145 | 0.306709 | 0.249137 | 0.223473 | 0.185947 |
| 20 | 2023-10-13 | 0.131816 | 0.154035 | 0.225009 | 0.131973 | 0.137236 | 0.070659 | 0.026579 | 0.064938 | -0.003257 |
| 21 | 2023-10-14 | 0.200610 | 0.267801 | 0.222852 | 0.250773 | 0.230686 | 0.252358 | 0.116113 | 0.220819 | 0.064967 |
| 22 | 2023-10-15 | 0.181522 | 0.219067 | 0.192574 | 0.274661 | 0.226209 | 0.140444 | 0.076887 | 0.129147 | 0.088821 |
| 23 | 2023-10-16 | 0.179420 | 0.209414 | 0.232633 | 0.256211 | 0.203172 | 0.187722 | 0.257174 | 0.165209 | 0.136125 |
| 24 | 2023-10-17 | 0.192814 | 0.198803 | 0.229261 | 0.268127 | 0.147232 | 0.298106 | 0.138862 | 0.254128 | 0.190610 |
| 25 | 2023-10-18 | 0.191258 | 0.202635 | 0.200670 | 0.332800 | 0.234716 | 0.180917 | 0.157915 | 0.141489 | 0.126197 |
| 26 | 2023-10-19 | 0.125263 | 0.125340 | 0.174479 | 0.155649 | 0.202877 | 0.137405 | 0.134489 | 0.214591 | 0.166119 |
| 27 | 2023-10-20 | 0.109986 | 0.172756 | 0.117033 | 0.113169 | 0.137168 | 0.212421 | 0.174950 | 0.158424 | 0.247471 |
| 28 | 2023-10-21 | 0.267251 | 0.264075 | 0.266374 | 0.239609 | 0.218864 | 0.225566 | 0.161354 | 0.198722 | 0.185505 |
| 29 | 2023-10-22 | 0.237799 | 0.199004 | 0.225758 | 0.263953 | 0.253231 | 0.243639 | 0.234858 | 0.233367 | 0.161174 |
| ds | unique_id | y | |
|---|---|---|---|
| 0 | 2023-10-11 | RNN | 0.135751 |
| 1 | 2023-10-12 | RNN | 0.069040 |
| 2 | 2023-10-13 | RNN | 0.035342 |
| 3 | 2023-10-14 | RNN | 0.191986 |
| 4 | 2023-10-15 | RNN | 0.166608 |
| ... | ... | ... | ... |
| 265 | 2023-10-18 | NBEATSx | 0.126197 |
| 266 | 2023-10-19 | NBEATSx | 0.166119 |
| 267 | 2023-10-20 | NBEATSx | 0.247471 |
| 268 | 2023-10-21 | NBEATSx | 0.185505 |
| 269 | 2023-10-22 | NBEATSx | 0.161174 |
270 rows × 3 columns